from copy import deepcopy
import scrapy
from scrapy import Selector
from spider_qunaer import items
from warehouse import models


class QunaerSpider(scrapy.Spider):
    name = 'qunaer'
    start_urls = ['https://travel.qunar.com/p-cs298890-xian-jingdian']
    page_num = 1

    def parse(self, response):
        item_scenery = items.SpiderSceneryItem()
        scenery_list = response.xpath("//ul[@class='list_item clrfix']/li")

        for i in scenery_list:
            item_scenery["scenery_name"] = i.xpath("./div/div/a/span/text()").extract_first()
            item_scenery["rank"] = i.xpath("./div/div/div/span[2]/span/text()").extract_first()
            item_scenery["people_percent"] = i.xpath("./div/div[2]/span/span/text()").extract_first()

            scenery = models.Scenery.objects.filter(scenery_name=item_scenery["scenery_name"]).first()
            if scenery:
                scenery.evaluates.all().delete()
                scenery.delete()

            detail_url = i.xpath("./a/@href").extract_first()
            if not models.Scenery.objects.filter(scenery_name=item_scenery["scenery_name"]).first():
                yield scrapy.Request(detail_url, callback=self.get_detail, encoding="utf-8", dont_filter=True,
                                     meta={"item_scenery": deepcopy(item_scenery)})

        if self.page_num < 500:
            self.page_num += 1
            yield scrapy.Request(url=f"{self.start_urls[0]}-1-{self.page_num}", callback=self.parse)

    def get_detail(self, response):
        item_scenery = response.meta["item_scenery"]

        # 检查响应类型
        if response.headers.get('Content-Type') == b'application/json':
            data = response.json()  # 解析JSON响应
            item_scenery["score"] = data.get("score", 0)  # 根据实际JSON结构更新
            item_scenery["play_time"] = data.get("play_time", None)
            item_scenery["city"] = data.get("city", "")
        else:
            # 对于HTML响应，使用XPath提取数据
            score = response.xpath(
                '//*[@id="js_mainleft"]/div[4]/div/div[2]/div[1]/div[1]/span[1]/text()').extract_first()
            item_scenery["score"] = float(score) if score else 0
            play_time = response.xpath('//div[@class="time"]/text()').extract_first()
            item_scenery["play_time"] = play_time.split("：")[1] if play_time else None
            item_scenery["city"] = response.xpath('//td[@class="td_l"]/dl[1]/dd/span/text()').extract_first()

        yield item_scenery

        # 第一页评论
        self.get_evalute(response)

        for path in response.xpath("//div[@class='b_paging']/a")[:50]:  # 取前50个评论链接
            evalute_path = path.xpath("./@href").extract_first()
            if evalute_path:
                yield scrapy.Request(evalute_path, callback=self.get_evalute, encoding="utf-8", dont_filter=True,
                                     meta={"item_scenery": deepcopy(item_scenery)})

    def get_evalute(self, response):
        item_scenery = response.meta["item_scenery"]

        # 检查是否为JSON响应
        if response.headers.get('Content-Type') == b'application/json':
            data = response.json()  # 解析JSON响应
            for evalute in data.get("evaluates", []):  # 假设 JSON 中包含 'evaluates' 键
                item_evalute = items.SpiderEvaluteItem()
                item_evalute["content"] = evalute.get("content", "").replace("阅读全部", "").replace("\n", "").replace(
                    "\r", "")
                item_evalute['send_time'] = evalute.get("send_time", "")
                item_evalute['user_name'] = evalute.get("user_name", "")
                item_evalute['score'] = evalute.get("score", 0)
                item_evalute['scenery_name'] = item_scenery['scenery_name']

                yield item_evalute
                item_scenery.instance.evaluates.add(item_evalute.instance)

        else:
            evalute_list = response.xpath("//ul[@id='comment_box']/li")
            if not evalute_list:
                return None
            for evalute in evalute_list:
                item_evalute = items.SpiderEvaluteItem()
                item_evalute["content"] = evalute.xpath("./div[1]/div[1]/div[@class='e_comment_content']").xpath(
                    'string(.)').extract_first().replace("阅读全部", "").replace("\n", "").replace("\r", "")
                item_evalute['send_time'] = evalute.xpath("./div[1]/div[1]/div[5]/ul/li[1]/text()").extract_first()
                item_evalute['user_name'] = evalute.xpath("./div[2]/div[2]/a/text()").extract_first()
                score = evalute.xpath("./div[1]/div[1]/div[2]/span/span/@class").extract_first()
                if score:
                    score = score.split("star_")[-1]
                item_evalute['score'] = score if score else 0
                item_evalute['scenery_name'] = item_scenery['scenery_name']

                yield item_evalute
                item_scenery.instance.evaluates.add(item_evalute.instance)
